#!/bin/bash

# exit on error
set -e

VERSION='v1.10.2a2-osv-port'  # dcccfbb2
BASEDIR="$PWD"
SRCDIR="$BASEDIR/ompi-release"

# Check for required build dependencies - libnuma-devel and libibverbs-devel.
# Just search for .h file installed by specific lib.
FF='numaif.h verbs.h'
for ff in $FF; do
    ffpath="`find /usr/include -iname $ff`"
    if [ -z "$ffpath" ]; then
        echo -e "Missing required devel libraries - libnuma-devel and libibverbs-devel.
To install, run:
sudo apt-get install libnuma-dev libibverbs-dev  # Ubuntu
sudo yum install numactl-devel libibverbs-devel  # CentOS
" 1>&2
        exit 1
    fi
done

CONFIGURE_FLAGS=""
DISTRO_NAME=`python -c 'import platform; pp=platform.linux_distribution(); str="%s-%s" % (pp[0], pp[1]); str=str.replace(" ", "-").lower(); print(str); '`
if [ "$mode" = "debug" ]; then
    CONFIGURE_FLAGS="--enable-debug"
    BUILD_DIR="build-osv-debug-$DISTRO_NAME"
else  # release
    CONFIGURE_FLAGS="--with-platform=optimized"
    BUILD_DIR="build-osv-release-$DISTRO_NAME"
fi
echo "Using Open MPI CONFIGURE_FLAGS=$CONFIGURE_FLAGS"

# enable parallel build
export MAKEFLAGS="-j `nproc`"

if [ ! -d ompi-release ]; then
    # Checkout code
    git clone --depth 1 --branch $VERSION https://github.com/mikelangelo-project/ompi-release.git
fi


mkdir -p $HOME/openmpi-bin
if [ ! -f ompi-release/$BUILD_DIR/AAA-build-ok ]
then
    # Configure and compile
    export AUTOMAKE_JOBS=`nproc`
    (cd ompi-release && ./autogen.pl)
    #
    mkdir -p ompi-release/$BUILD_DIR
    cd ompi-release/$BUILD_DIR
    #
    # OSv chrashes at VM termination if SYSV shmem is used. Disable it until fixed.
    # Otherwise, SYSV shmem seems to work.
    # See https://github.com/cloudius-systems/osv/issues/755.
    ../configure --prefix=$HOME/openmpi-bin --disable-dlopen \
        --disable-oshmem --disable-posix-shmem \
        --disable-sysv-shmem \
        --enable-mpi-fortran \
        CFLAGS="-fPIC -DPIC" --disable-pty-support --disable-vt \
        --enable-static --enable-shared $CONFIGURE_FLAGS
    make -j `nproc`
    chmod a+x ../make-orted-so.sh  # TODO fix git
    ../make-orted-so.sh

    # No need to install Open MPI to get it compiled for OSv.
    # But mpicc is needed to compile OpenFOAM, so we install it anyway.
    #
    # To compile OpenFOAM, it would be enough to install system supplied
    # Open MPI, but then a simple 'mpirun ...' would use wrong binary.
    # This would require additional messing with PATH/LD_LIBRARY_PATH.
    make install  # safe with -prefix=$HOME/openmpi-bin
    # add to $PATH, $LD_LIBRARY_PATH ...
    if [ -z "`cat ~/.bashrc | grep 'MAGIC LINE mike-apps open-mpi included'`" ]; then
        cat <<EOF >> ~/.bashrc

# MAGIC LINE mike-apps open-mpi included : OSv mike-apps openmpi bin/libs
export PATH="$HOME/openmpi-bin/bin:\$PATH"
export LD_LIBRARY_PATH="$HOME/openmpi-bin/lib:\$LD_LIBRARY_PATH"

EOF
    fi
    cd $BASEDIR # go back to module dir
    date >> AAA-build-ok
fi

# even if user removes in ~/.bashrc added PATH/LD_LIBRARY_PATH entries, we need them in this script
# so that ldd is able to find shared library dependencies. So set them once more. Plus, on first run,
# ~/.bashrc is updated, but we haven't re-source it.
export PATH="$HOME/openmpi-bin/bin:$PATH"
export LD_LIBRARY_PATH="$HOME/openmpi-bin/lib:$LD_LIBRARY_PATH"

OMPI_LIBS=""
OMPI_LIBS+=" ompi/.libs/libmpi.so.12"
OMPI_LIBS+=" opal/.libs/libopen-pal.so.13"
OMPI_LIBS+=" orte/.libs/libopen-rte.so.12"
OMPI_LIBS+=" ompi/mpi/cxx/.libs/libmpi_cxx.so.1"
OMPI_LIBS+=" orte/tools/orted/.libs/orted.so"
OMPI_LIBS+=" orte/tools/orterun/.libs/orterun.so"
# Fortran libs
# There are multiple libraries under ompi/mpi/fortran/, and it is not obvious
# which libs will be selected by ./configure, so include all compiled libs.
fortran_libs=`cd ompi-release/$BUILD_DIR && find . -iname '*\.so\.[0-9]' -o -iname '*\.so\.[0-9][0-9]'`
OMPI_LIBS+=" $fortran_libs "

# Open MPI programs and base set of libraries
echo "" > "$BASEDIR/usr.manifest"
for ff in $OMPI_LIBS
do
    LIB_NAME="`basename "$ff"`"
    echo "/usr/lib/$LIB_NAME:    ${SRCDIR}/$BUILD_DIR/$ff" >> "$BASEDIR/usr.manifest"
done

# Add one additional symlink (mpirun is actually orterun).
# Also, some apps are linked against libmpi.so.1, not libmpi.so.12.
# This substitute seems to work.
echo "" >> "$BASEDIR/usr.manifest"
echo "/usr/lib/libmpi.so.1:    ->/usr/lib/libmpi.so.12" >> "$BASEDIR/usr.manifest"
echo "/usr/bin/mpirun:    ->/usr/lib/orterun.so" >> "$BASEDIR/usr.manifest"

# To test fortran support, uncomment lines below. After rebuild, run VM as:
#    ./scripts/run.py -dV -e "/usr/bin/mpirun --allow-run-as-root -H localhost -np 2 /hello_mpifh.so"
#(cd  ${SRCDIR}/examples/ && mpifort -g hello_mpifh.f -o hello_mpifh.so -fPIC -shared)
#echo "/hello_mpifh.so:    ${SRCDIR}/examples/hello_mpifh.so" >> "$BASEDIR/usr.manifest"
#OMPI_LIBS+=" ../examples/hello_mpifh.so "  # to include its dependencies

# Add Open MPI help files.
# make install puts them into $HOME subdir, and that path is then baked into compiled binaries too.
echo "$HOME/openmpi-bin/share/openmpi/**: $HOME/openmpi-bin/share/openmpi/**" >> "$BASEDIR/usr.manifest"

# Include dependency libs, used by Open MPI progs/libs.
echo "" >> "$BASEDIR/usr.manifest"
DEP_LIBS=""
for ff in $OMPI_LIBS
do
    DESTF_FULL="${SRCDIR}/$BUILD_DIR/$ff"
    # Exclude some libs.
    # Simple programs work without libnl-3.so and libnl-route-3.so.
    # With them, complain about symbol tdestroy.
    # This seems to be part of glibc GNU extension, and is missing in OSv.
    CUR_DEP_LIBS="`ldd "$DESTF_FULL" | grep -Po '(?<=> )/[^ ]+' | sort | uniq`"
    DEP_LIBS+="`echo "$CUR_DEP_LIBS" | grep -Pv 'lib(c|gcc|dl|m|util|rt|pthread|stdc\+\+|mpi|mpi_cxx|open-pal|open-rte|nl-3|nl-route-3).so' || true`"
    DEP_LIBS+="
"
done
DEP_LIBS=`echo "$DEP_LIBS" | sort | uniq`
for ff in $DEP_LIBS
do
    lib_name=`basename "$ff"`
    echo "/usr/lib/$lib_name:    $ff" >> "$BASEDIR/usr.manifest"
done
